# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
import codecs
from logging import log
import pymongo
# from scrapy import log
from scrapy.conf import settings
from scrapy.exceptions import DropItem


class HaodfPipeline(object):
    def __init__(self):
        self.file = codecs.open('haodf_data.json', mode='wb', encoding='utf-8')

    def process_item(self, items, spider):
        line = json.dumps(dict(items)) + '\n'
        # print line
        self.file.write(line.decode("unicode_escape"))
        return items


class XywyPipeline(object):
    def __init__(self):
        self.file = codecs.open('xywy_data.json', mode='wb', encoding='utf-8')

    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + '\n'
        print line
        self.file.write(line.decode('unicode_escape'))
        return item


class MongoPipeline(object):
    # scrapy官方手册 http://doc.scrapy.org/en/latest/topics/item-pipeline.html?highlight=mongodb
    collection_name = 'url'
    # collection_name = 'scrapy_items'

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[self.collection_name].insert(dict(item))
        return item

class MongoDBPipeline(object):
    collection_name = 'article'
    # collection_name = 'scrapy_items'

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGODB_DB', 'items')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[self.collection_name].insert(dict(item))
        return item

    # chm手册中的方式，测试不成功
    # def __init__(self):
    #     connection = pymongo.Connection(
    #         settings['MONGODB_SERVER'],
    #         settings['MONGODB_PORT']
    #     )
    #     db = connection[settings['MONGODB_DB']]
    #     self.collection = db[settings['MONGODB_COLLECTION']]
    #
    # def process_item(self, item, spider):
    #     for field, data in item.items():
    #         if not data:
    #             raise DropItem('Missing data!')
    #     self.collection.update({'url': item['url']}, dict(item), upsert=True)
    #     log.msg('Item written to MongoDB database !', level=log.DEBUG, spider=spider)
    #     return item


class WenzhenPipeline(object):
    def process_item(self, item, spider):
        return item


